import re
import time
import json
import requests
from lxml import etree
url = 'https://www.walmart.com/all-departments'
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
}

src = requests.get(url, headers=headers, allow_redirects=False)
print(src.status_code)
category_json = re.compile('<script id="home" type="application/json">(.*?)</script>').findall(src.text)[0]
category_json = json.loads(category_json)
print(category_json)
print(type(category_json))
departments = category_json['home']['home']['quimbyData']['global_header_ny']['headerZone1']['configs']['departments']
retrieve_json = []
print('='*100)
for i in departments:
    aa = {i['name']: [{j['department']['title']: j['department']['clickThrough']['value']} for j in i['departments']]}
    print(aa)
    time.sleep(2)


















